#########################################################################
# File Name: collect_msg.sh
# Author: ****
# mail: ****.com
# Created Time: Wed Mar 27 14:56:03 2024
#########################################################################
#!/bin/bash
current_path=$(pwd)
config_file=$current_path/config.ini
log_path=$current_path/log
default_project="Bigdata Database Storage Arm Virtcca Kae Virtual"

spark_omni_func=(
    --deploy-mode client
    --driver-cores 1
    --driver-memory 980M
    --num-executors 3
    --executor-cores 1
    --executor-memory 600M
    --master yarn
    --conf spark.memory.offHeap.enabled=true
    --conf spark.memory.offHeap.size=1025M
    --conf spark.task.cpus=1
    --conf spark.driver.extraClassPath=/opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar:/opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar:/opt/omni-operator/lib/dependencies/*
    --conf spark.executor.extraClassPath=/opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar:/opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar:/opt/omni-operator/lib/dependencies/*
    --driver-java-options -Djava.library.path=/opt/omni-operator/lib
    --conf spark.sql.codegen.wholeStage=false
    --conf spark.executorEnv.LD_LIBRARY_PATH=/opt/omni-operator/lib
    --conf spark.executorEnv.OMNI_HOME=/opt/omni-operator/
    --conf spark.driverEnv.LD_LIBRARY_PATH=/opt/omni-operator/lib
    --conf spark.driverEnv.OMNI_HOME=/opt/omni-operator/
    --conf spark.executor.extraLibraryPath=/opt/omni-operator/lib
    --conf spark.driverEnv.LD_PRELOAD=/opt/omni-operator/lib/libjemalloc.so.2
    --conf spark.executorEnv.LD_PRELOAD=/opt/omni-operator/lib/libjemalloc.so.2
    --conf spark.sql.extensions=com.huawei.boostkit.spark.ColumnarPlugin
    --jars /opt/omni-operator/lib/boostkit-omniop-spark-3.1.1-1.4.0-aarch64.jar
    --jars /opt/omni-operator/lib/boostkit-omniop-bindings-1.4.0-aarch64.jar
    --conf spark.sql.orc.impl=native
    --conf spark.shuffle.manager=org.apache.spark.shuffle.sort.OmniColumnarShuffleManager
    --conf spark.omni.sql.columnar.fusion=false
    --conf spark.omni.sql.columnar.sortSpill.enabled=true
    --conf spark.omni.sql.columnar.sortSpill.rowThreshold=4000000
    --conf spark.omni.sql.columnar.sortSpill.dirDiskReserveSize=214748364800
    --conf spark.locality.wait=8
    --conf spark.sql.autoBroadcastJoinThreshold=10M
    --comf spark.sql.broadcastTimeout=500
    --conf spark.sql.cbo.enabled=false
    --conf spark.default.parallelism=200
    --conf spark.sql.shuffle.partitions=200
    --conf spark.executorEnv.MALLCO_CONF=narenas:2
)

database='tpcds_bin_partitioned_varchar_orc_2'

#####################根据section以及key值获取对应value############
acquire_value(){
    project=$1
    key=$2
    grep $project -A 10  $config_file |grep -m 1 $key|awk -F= '{print $2}'|awk '{print $1}'
}


######################获取配置文件中选择校验的解决方案###########
acquire_select_project(){
    all_name=""
	for per_project in $default_project
	do
		status=$(acquire_value $per_project check)
		if [[ $status = True ]]; then
			all_name="$all_name $per_project"
        fi
	done
	echo $all_name
}

###################arm原生特性信息收集#################
collect_arm_native(){
     kbox_container=$1
     video_container=$2
     instruction_container=$3
     # kbox基础云手机
     rm -f $log_path/arm_native.log
     
     keyword=(gralloc.kbox.so audio.primary.kbox.so gps.kbox.so sensors.kbox.so libmedia_omxcore.so libstagefrighthw.so vinput hwcomposer.kbox.so)
     docker exec -it $kbox_container lsof > $log_path/arm_native_raw.log
     for i in "${keyword[@]}";do
	     grep -F "${i}" $log_path/arm_native_raw.log >> $log_path/arm_native.log
     done
     rm -f $log_path/arm_native_raw.log

     docker exec -it $kbox_container cat /proc/sys/fs/binfmt_misc/ubt_a32a64 >> $log_path/arm_native.log
     # 视频流
     docker exec -it $video_container lsof | grep VmiInputFlinger >> $log_path/arm_native.log
     # 指令流
     docker exec -it $instruction_container ps -ef | grep -F "VmiAgent instruction" >> $log_path/arm_native.log
}
#################虚拟化特性信息收集##################

collect_virtual(){
    ovs-appctl hwoff/dump-hw-flows |grep "No flow available" >> virtual_feautre.log
    sudo systemctl status waasagent.service >> virtual_feature.log
	waasctl --version >> virtual_feature.log
}

################数据库特性信息收集##################
collect_database(){
   mysql_install_path=$1
   mysql_username=$2
   mysql_password=$3
   database_name=$4
   plugin_path=$1/lib/plugin
   $mysql_install_path/bin/mysqld -u $mysql_username -p$mysql_password -D $database_name -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'thread_pool%'" > $log_path/database_mysql.log
   $mysql_install_path/bin/mysqld -u $mysql_username -p$mysql_password -D $database_name -e "select * from INFORMATION_SCHEMA.plugins where PLUGIN_NAME like 'kovae%'" >> $log_path/database_mysql.log
   echo thread_pool: $(ls $plugin_path |grep thread_pool.so) >> $log_path/database_mysql.log
   echo kovae_path: $(ls $plugin_path |grep ha_kovae.so) >>  $log_path/database_mysql.log
   readelf -a $mysql_install_path/bin/mysqld|grep bolt >> $log_path/database_mysql.log
   echo no_lock: $(objdump -d $mysql_install_path/bin/mysqld|grep -c row_vers_build_for_semi_consistent_readP5trx_t) >> $log_path/database_mysql.log
   objdump -d $mysql_install_path/bin/mysqld |grep crc32cb >> $log_path/database_mysql.log
}


####################机密计算特性信息收集################
collect_virtcca_msg(){
	cvm_name=$1
	username=$2
	passwd=$3
	xml_path=/tmp/temp.xml
	virsh list --all|grep -q $cvm_name
	if [ $? -ne 0 ];then
		echo "错误:虚拟机$cvm_name不存在"
		exit 1
	fi
	cvm_status=$(virsh domstate "$cvm_name")

	if [ "$cvm_status" == "shut off" ];then
		echo "虚拟机$cvm_name处于shut off状态,正在启动..."
		virsh start "$cvm_name"
		echo "虚拟机$cvm_name完成启动"
	elif ["$cvm_status" == "running" ];then
		echo "虚拟机$cvm_name处于running状态,无需操作"
	else
		echo "错误:无法确定虚拟机$cvm_status的状态
		exit 1
	fi

	virsh dumpxml $cvm_name > $xml_path
	ret=$(grep -i "type='cvm'" $xml_path)
	echo "$ret" > $log_path/virtcca_status.log
	expect << EOF >> $log_path/virtcca_status.log
	spawn virsh cosole $cvm_name
	expect "Escape character is \\^]"
	send "\r"
	expect "login:"
	send "$username\r"
	expect "Password:"
	send "$passwd\r"
	expect "# "
	send "ls -l /\r"
	expect "# "
	send "exit\r"
	expect eof
EOF
}


#################加速库特性信息收集##################

collect_acceleration_library(){
    system_lib=$1
    hmpp_lib=$2
    math_lib=$3
    openssl speed -elapsed -engine kae rsa2048 |grep "invalid engine" >> acceleration_library.log
    ldd $1 >> acceleration_library.log
    ldd $2 >> acceleration_library.log
    ldd $3 >> acceleration_library.log
}


#################HPC特性信息收集##################

#hpc 加速库

#判断是否使用加速库，使用加速库超过三个即视为通过
collect_acceleration_library(){
    
    bin_path=$1 #用户可执行文件路径
    rm -rf acceleration_hpc_library.log
    touch acceleration_hpc_library.log
    
    ldd $bin_path | grep hypero >> acceleration_hpc_library.log
    ldd $bin_path | grep libkm_l >>  acceleration_hpc_library.log
    ldd $bin_path | grep hmpi >>  acceleration_hpc_library.log
    ldd $bin_path | grep tydss >>  acceleration_hpc_library.log

}


collect_acceleration_library(){

    ifsme=`lscpu|grep Flags|grep sme`
    if [ -n "$ifsme" ]; then
        bin_path=$1 #用户可执行文件路径
        rm -rf SME_hpc_library.log
        touch SME_hpc_library.log    
        ldd $bin_path | grep SME >>SME_hpc_library.log
        num=`cat SME_hpc_library.log|wc -l`
        objdump -d $bin_path > $bin_path.asm
        for i in `cat SME_INSTRUCTION`
        do
            cat SME_hpc_library.log|grep $i

        done
    fi



}



###############分布式存储特性信息收集###############

collect_storage(){
    ldd /usr/bin/ceph-osd > storage.log
    bcache1=$(ls /sys/class/block|grep -m 1 bcache)
    ll /sys/class/block/bcache0/bcache/cache/internal/traffic_policy_start >> storage.log
}


collect_storage_ksal(){
    ec_pool=$1
    pool_list=$(rados lspools |grep -wx $ec_pool)
    if [[ $pool_list =~ $ec_pool  ]];
    then
	    echo "ec_pool created" > storage_ksal.log
	    pid_num=$(ps -ef|grep  osd|grep -v grep|head -n 1|awk '{print $2}')
	    cat /proc/$pid_num/smaps |grep ksal >> storage_ksal.log
    else
	    echo "ec_pool not exist"> storage_ksal.log
    fi

}


###############大数据特性信息收集##################
collect_kal(){
  algotithm_list=$1
  algotithm_path=$2
  dataset_list=$3

  read -r -a  algotithm_arry <<< "$algotithm_list"
  read -r -a  dataset_arry <<< "$dataset_list"
  
  index=0
  for per_alg in algotithm_list
  do  
      
      bash $algotithm_path/bin/ml/${algotithm_arry[$index]}_run.sh ${dataset_arry[$index]} no no > bigdata_kal_${algotithm_arry[$index]}_${dataset_arry[$index]}.log 2>&1
  done
  
  

}


collect_operator(){
        # 日志记录位置 log_path/bigdata_operator.log
	spark_path=$1
        spark_conf_path=$1/conf
	if ! cat < $spark_conf_path/log4j.properties|grep "^log4j.logger.com.huawei.boostkit.spark=INFO";
	then
           echo "log4j.logger.com.huawei.boostkit.spark=INFO" >> $spark_conf_path/log4j.properties
	fi
	$SPARK_HOME/bin/spark-sql "${spark_omni_func[@]}" --database $database -e "WITH customer_total_return AS ( SELECT sr_customer_sk AS ctr_customer_sk, sr_store_sk AS ctr_store_sk, sum(sr_return_amt) AS ctr_total_return FROM store_returns, date_dim WHERE sr_returned_date_sk = d_date_sk AND d_year = 2000 GROUP BY sr_customer_sk, sr_store_sk) SELECT c_customer_id FROM customer_total_return ctr1, store, customer WHERE ctr1.ctr_total_return > (SELECT avg(ctr_total_return) * 1.2 FROM customer_total_return ctr2 WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) AND s_store_sk = ctr1.ctr_store_sk AND s_state = 'TN' AND ctr1.ctr_customer_sk = c_customer_sk ORDER BY c_customer_id LIMIT 100;" 1>$log_path/bigdata_operator.log 2>&1

}


collect_hbase(){

   hbase com.huawei.boostkit.hindex.mapreduce.GlobalTableIndexer-Dtablename.to.index=OnlySingleIndexTbale-Dindexspecs.to.addandbuild='osindex=>C0:[F1]'
   hbase shell << EOF > $log_path/bigdata_hbase.log 2>&1
   debug
   scan 'OnlySingleIndexTable',{FILTER=>"(SingleColumnValueFilter('C0', 'F1',=,'binary:bbb')"} 
   exit
EOF 
}


collect_tune_up(){
   omniadvisor_dir=$1
   mysql_username=$2
   mysql_password=$3
   mysql_database=$4
   if [ -e $omniadvisor_dir/omniadvisor ];
   then
       echo "omniadvisor.log" >> $log_path/bigdata_tune_up.log
   else
      echo "omniadvisor.log not exist" >> $log_path/bigdata_tune_up.log
   fi
   
   mysql -u $mysql_username -p$mysql_password -D $mysql_database  -e "show tables" >> $log_path/bigdata_tune_up.log

}


main(){
    if [ -e $config_file ]; then
		select_project=$(acquire_select_project)
		echo "开始收集BoostKit 特性信息如下：$select_project"
		mkdir -p $log_path
		echo "日志存放位置： $log_path"
	else
		echo "config.ini not exist"
        exit 1
	fi
    
	for per_project in $select_project
	do
		if [ $per_project = "Bigdata" ];
		then
			echo "start collect Bigdata msg..."
	    elif [ $per_project = "Virtcca" ];
		then
			echo "start collect virtcca msg..."
			cvm_name=$(acquire_value Virtcca cvm_name)
                        cvm_username=$(acquire_value Virtcca cvm_username)
                        cvm_password=$(acquire_value Virtcca cvm_password)
                        collect_virtcca_msg $cvm_name $cvm_username $cvm_password
		elif [ $per_project = "Database" ];
		then
		    echo "start collect database msg..."
                    collect_database $(acquire_value Database mysql_install_path)

		fi
    done
}
main
